package com.gy.spark.core

import org.apache.spark.{SparkConf, SparkContext}
/*
spark-submit \
--master spark://centos101:7077,centos102:7077 \
--total-executor-cores 1 \
--executor-memory 1G \
--class com.gy.spark.WordCountCluster \
kkb-bigdata-1.0-SNAPSHOT.jar /words.txt /output/spark/wc
 */
/**
 * 集群模式运行wc
 *
 */
object WordCountCluster {

  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf()
      .setAppName(WordCountCluster.getClass.getSimpleName)

    val sc = new SparkContext(sparkConf)

    sc.setLogLevel("warn")

    sc.textFile(args(0))
      .flatMap(_.split(" "))
      .map((_, 1))
      .reduceByKey(_ + _)
      .sortBy(_._2, false)
      .saveAsTextFile(args(1))

    sc.stop()

  }


}
